{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# python\n",
    "class Node(object):\n",
    "    \"\"\"\n",
    "    Base class for nodes in the network.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `inbound_nodes`: A list of nodes with edges into this node.\n",
    "    \"\"\"\n",
    "    def __init__(self, inbound_nodes=[]):\n",
    "        \"\"\"\n",
    "        Node's constructor (runs when the object is instantiated). Sets\n",
    "        properties that all nodes need.\n",
    "        \"\"\"\n",
    "        # A list of nodes with edges into this node.\n",
    "        self.inbound_nodes = inbound_nodes\n",
    "        # The eventual value of this node. Set by running\n",
    "        # the forward() method.\n",
    "        self.value = None\n",
    "        # A list of nodes that this node outputs to.\n",
    "        self.outbound_nodes = []\n",
    "        # New property! Keys are the inputs to this node and\n",
    "        # their values are the partials of this node with\n",
    "        # respect to that input.\n",
    "        self.gradients = {}\n",
    "\n",
    "        # Sets this node as an outbound node for all of\n",
    "        # this node's inputs.\n",
    "        for node in inbound_nodes:\n",
    "            node.outbound_nodes.append(self)\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Every node that uses this class as a base class will\n",
    "        need to define its own `forward` method.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Every node that uses this class as a base class will\n",
    "        need to define its own `backward` method.\n",
    "        \"\"\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "\n",
    "class Input(Node):\n",
    "    \"\"\"\n",
    "    A generic input into the network.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        Node.__init__(self)\n",
    "\n",
    "    def forward(self):\n",
    "        pass\n",
    "\n",
    "    def backward(self):\n",
    "        self.gradients = {self: 0}\n",
    "        for n in self.outbound_nodes:\n",
    "            self.gradients[self] += n.gradients[self]\n",
    "\n",
    "class Linear(Node):\n",
    "    \"\"\"\n",
    "    Represents a node that performs a linear transform.\n",
    "    \"\"\"\n",
    "    def __init__(self, X, W, b):\n",
    "        Node.__init__(self, [X, W, b])\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Performs the math behind a linear transform.\n",
    "        \"\"\"\n",
    "        X = self.inbound_nodes[0].value\n",
    "        W = self.inbound_nodes[1].value\n",
    "        b = self.inbound_nodes[2].value\n",
    "        self.value = np.dot(X, W) + b\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient based on the output values.\n",
    "        \"\"\"\n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}\n",
    "        for n in self.outbound_nodes:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.inbound_nodes[0]] += np.dot(grad_cost, self.inbound_nodes[1].value.T)\n",
    "            self.gradients[self.inbound_nodes[1]] += np.dot(self.inbound_nodes[0].value.T, grad_cost)\n",
    "            self.gradients[self.inbound_nodes[2]] += np.sum(grad_cost, axis=0, keepdims=False)\n",
    "\n",
    "\n",
    "class Sigmoid(Node):\n",
    "    \"\"\"\n",
    "    Represents a node that performs the sigmoid activation function.\n",
    "    \"\"\"\n",
    "    def __init__(self, node):\n",
    "        Node.__init__(self, [node])\n",
    "\n",
    "    def _sigmoid(self, x):\n",
    "        \"\"\"\n",
    "        This method is separate from `forward` because it\n",
    "        will be used with `backward` as well.\n",
    "\n",
    "        `x`: A numpy array-like object.\n",
    "        \"\"\"\n",
    "        return 1. / (1. + np.exp(-x))\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Perform the sigmoid function and set the value.\n",
    "        \"\"\"\n",
    "        input_value = self.inbound_nodes[0].value\n",
    "        self.value = self._sigmoid(input_value)\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient using the derivative of\n",
    "        the sigmoid function.\n",
    "        \"\"\"\n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inbound_nodes}\n",
    "        for n in self.outbound_nodes:\n",
    "            grad_cost = n.gradients[self]\n",
    "            sigmoid = self.value\n",
    "            self.gradients[self.inbound_nodes[0]] += sigmoid * (1 - sigmoid) * grad_cost\n",
    "\n",
    "\n",
    "class MSE(Node):\n",
    "    def __init__(self, y, a):\n",
    "        \"\"\"\n",
    "        The mean squared error cost function.\n",
    "        Should be used as the last node for a network.\n",
    "        \"\"\"\n",
    "        Node.__init__(self, [y, a])\n",
    "\n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Calculates the mean squared error.\n",
    "        \"\"\"\n",
    "        y = self.inbound_nodes[0].value.reshape(-1, 1)\n",
    "        a = self.inbound_nodes[1].value.reshape(-1, 1)\n",
    "\n",
    "        self.m = self.inbound_nodes[0].value.shape[0]\n",
    "        self.diff = y - a\n",
    "        self.value = np.mean(self.diff**2)\n",
    "\n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Calculates the gradient of the cost.\n",
    "        \"\"\"\n",
    "        self.gradients[self.inbound_nodes[0]] = (2 / self.m) * self.diff\n",
    "        self.gradients[self.inbound_nodes[1]] = (-2 / self.m) * self.diff\n",
    "        \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# python\n",
    "def topological_sort(feed_dict):\n",
    "    \"\"\"\n",
    "    Sort the nodes in topological order using Kahn's Algorithm.\n",
    "\n",
    "    `feed_dict`: A dictionary where the key is a `Input` Node and the value is the respective value feed to that Node.\n",
    "\n",
    "    Returns a list of sorted nodes.\n",
    "    \"\"\"\n",
    "    input_nodes = [n for n in feed_dict.keys()]\n",
    "    G = {}\n",
    "    nodes = [n for n in input_nodes]\n",
    "    while len(nodes) > 0:\n",
    "        n = nodes.pop(0)\n",
    "        if n not in G:\n",
    "            G[n] = {'in': set(), 'out': set()}\n",
    "        for m in n.outbound_nodes:\n",
    "            if m not in G:\n",
    "                G[m] = {'in': set(), 'out': set()}\n",
    "            G[n]['out'].add(m)\n",
    "            G[m]['in'].add(n)\n",
    "            nodes.append(m)\n",
    "\n",
    "    L = []\n",
    "    S = set(input_nodes)\n",
    "    while len(S) > 0:\n",
    "        n = S.pop()\n",
    "        if isinstance(n, Input):\n",
    "            n.value = feed_dict[n]\n",
    "\n",
    "        L.append(n)\n",
    "        for m in n.outbound_nodes:\n",
    "            G[n]['out'].remove(m)\n",
    "            G[m]['in'].remove(n)\n",
    "            if len(G[m]['in']) == 0:\n",
    "                S.add(m)\n",
    "    return L\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forward_and_backward(graph):\n",
    "    \"\"\"\n",
    "    Performs a forward pass and a backward pass through a list of sorted Nodes.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `graph`: The result of calling `topological_sort`.\n",
    "    \"\"\"\n",
    "    for n in graph:\n",
    "        n.forward()\n",
    "\n",
    "    for n in graph[::-1]:\n",
    "        n.backward()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sgd_update(trainables, learning_rate=1e-2):\n",
    "    \"\"\"\n",
    "    Updates the value of each trainable with SGD.\n",
    "\n",
    "    Arguments:\n",
    "\n",
    "        `trainables`: A list of `Input` Nodes representing weights/biases.\n",
    "        `learning_rate`: The learning rate.\n",
    "    \"\"\"\n",
    "    for t in trainables:\n",
    "        t.value = t.value - learning_rate * t.gradients[t]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建鸢尾花分类数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample\n",
    "from sklearn import datasets\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "data = datasets.load_iris()\n",
    "X_ = data.data\n",
    "y_ = data.target\n",
    "y_[y_==2] = 1 # 0 for virginica, 1 for not virginica\n",
    "print(X_.shape, y_.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 搭建深度神经网络模型\n",
    "\n",
    "可以理解为两层逻辑斯蒂回归串联，输入层4个特征，输出层2个预测（不是 virginica 的可能性），中间层这里设置为3。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "n_features = X_.shape[1]\n",
    "n_class = 1\n",
    "n_hidden = 3\n",
    "\n",
    "\n",
    "X, y = Input(), Input()\n",
    "W1, b1 = Input(), Input()\n",
    "W2, b2 = Input(), Input()\n",
    "\n",
    "l1 = Linear(X, W1, b1)\n",
    "s1 = Sigmoid(l1)\n",
    "l2 = Linear(s1, W2, b2)\n",
    "t1 = Sigmoid(l2)\n",
    "cost = MSE(y, t1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 随机初始化参数值\n",
    "W1_0 = np.random.random(X_.shape[1]*n_hidden).reshape([X_.shape[1], n_hidden])\n",
    "W2_0 = np.random.random(n_hidden*n_class).reshape([n_hidden, n_class])\n",
    "b1_0 = np.random.random(n_hidden)\n",
    "b2_0 = np.random.random(n_class)\n",
    "\n",
    "# 将输入值带入算子\n",
    "feed_dict = {\n",
    "    X: X_,   y: y_,\n",
    "    W1: W1_0, b1: b1_0,\n",
    "    W2: W2_0, b2: b2_0\n",
    "}\n",
    "\n",
    "# 训练参数\n",
    "# 这里训练100轮（eprochs），每轮抽4个样本（batch_size）训练150/4次（steps_per_eproch）,学习率 0.1\n",
    "epochs = 100\n",
    "m = X_.shape[0]\n",
    "batch_size = 4\n",
    "steps_per_epoch = m // batch_size\n",
    "lr = 0.1\n",
    "\n",
    "graph = topological_sort(feed_dict)\n",
    "trainables = [W1, b1, W2, b2]\n",
    "\n",
    "l_Mat_W1 = [W1_0]\n",
    "l_Mat_W2 = [W2_0]\n",
    "\n",
    "l_loss = []\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "    for j in range(steps_per_epoch):\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "        \n",
    "        forward_and_backward(graph)\n",
    "        sgd_update(trainables, lr)\n",
    "        loss += graph[-1].value\n",
    "    \n",
    "    l_loss.append(loss)\n",
    "    if i % 10 == 9:\n",
    "        print(\"Eproch %d, Loss = %1.5f\" % (i, loss))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(l_loss)\n",
    "plt.title(\"Cross Entropy value\")\n",
    "plt.xlabel(\"Eproch\")\n",
    "plt.xlabel(\"Loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用模型预测所有数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X.value = X_\n",
    "y.value = y_\n",
    "for n in graph:\n",
    "    n.forward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(graph[-2].value.ravel())\n",
    "plt.title(\"Predict for all 150 Iris data\")\n",
    "plt.xlabel(\"Sample ID\")\n",
    "plt.ylabel(\"Probability for not a virginica\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
